"""Documentation about document object"""

import nltk.data
sentence_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

#Gunakan WordPunctTokenizar agar dipisah berdasarkan punctuation
from nltk.tokenize import WordPunctTokenizer
word_tokenizer = WordPunctTokenizer()

class Document:
    def __init__(self, id, title, author, content):
        self.id = id
        self.title = title
        self.author = author
        self.content = content
    
    def __str__(self):
        return "id: %stitle: %sauthor: %scontent: %s" %(self.id, self.title, self.author, self.content)
    
    def lowerCase(self):
        self.id = self.id.lower()
        self.title = self.title.lower()
        self.author = self.author.lower()
        self.content = self.content.lower()
    
    def titleTokenization(self):
        #Because title is only one sentence, just use word level tokenization
        title_list_token = word_tokenizer.tokenize(self.title)
        return title_list_token
        
    def authorTokenization(self):
        #Because author is separated by new line, we use standard splitting
        author_list_token = self.author.split('\n')
        if author_list_token[-1] == "":
            del author_list_token[-1]
        return author_list_token
    
    def contentTokenization(self):
        #Because content of document is paragraph, we first use sentence tokenize
        sentence_list_token = sentence_tokenizer.tokenize(self.content)
        #For each sentence, we tokenize each sentence become word
        content_list_token = []
        for sentence in sentence_list_token:
            content_list_token += word_tokenizer.tokenize(sentence)
        
        #content_list_token = [helper.punctuation.sub("", word) for word in content_list_token]
        #word_list = [punctuation.sub("", word) for word in word_list]    
        return content_list_token


if __name__ == '__main__':
    id = """1
"""
    title = """the ibm dsd technical information center - a total systems approach
combining traditional library features
and mechanized computer processing
"""
    author = """H. S. WHITE
"""
    content = """the ibm data systems division technical
 information center (tic) provides an operating developmental
system for integrated and compatible mechanized
 processing of technical information received within the organization.
  the system offers several advantages :
     1 . it is a sophisticated mechanized system for dissemination
and retrieval;
     2 . it is compatible with all library mechanized
  records produced under a standard processing format
  within ibm libraries, providing such traditional tools
  as 3 x 5 catalog cards, circulation records and overdue
notices;
     3 . it is reversible, so that discontinuation of machine
processing would not cause gaps in the library's
  manual records;
     4 . it is controlled, producing statistical evaluations
of its own program efficiency;
     5 . it is user-oriented, providing 24-hour copy access
and immediate microfilm access to its documents;
     6 . it is relatively simple, relying on the ibm 1401
  data processing system for all its processing and output.

  since the system has been operating for over a year, the
conclusions drawn are based on actual experience .
"""
    doc = Document(id, title, author, content)
    doc.lowerCase()
    #print doc
    print doc.titleTokenization()
    print doc.authorTokenization()
    print doc.contentTokenization()


#['the', 'ibm', 'data', 'systems', 'division', 'technical', 'information', 'center', '', 'tic', '', 'provides', 'an', 'operating', 'developmental', 'system', 'for', 'integrated', 'and', 'compatible', 'mechanized', 'processing', 'of', 'technical', 'information', 'received', 'within', 'the', 'organization', '', 'the', 'system', 'offers', 'several', 'advantages', '', '', '', 'it', 'is', 'a', 'sophisticated', 'mechanized', 'system', 'for', 'dissemination', 'and', 'retrieval', '', '', '', 'it', 'is', 'compatible', 'with', 'all', 'library', 'mechanized', 'records', 'produced', 'under', 'a', 'standard', 'processing', 'format', 'within', 'ibm', 'libraries', '', 'providing', 'such', 'traditional', 'tools', 'as', '', 'x', '', 'catalog', 'cards', '', 'circulation', 'records', 'and', 'overdue', 'notices', '', '', '', 'it', 'is', 'reversible', '', 'so', 'that', 'discontinuation', 'of', 'machine', 'processing', 'would', 'not', 'cause', 'gaps', 'in', 'the', 'library', "'", 's', 'manual', 'records', '', '', '', 'it', 'is', 'controlled', '', 'producing', 'statistical', 'evaluations', 'of', 'its', 'own', 'program', 'efficiency', '', '', '', 'it', 'is', 'user', '', 'oriented', '', 'providing', '', '', 'hour', 'copy', 'access', 'and', 'immediate', 'microfilm', 'access', 'to', 'its', 'documents', '', '', '', 'it', 'is', 'relatively', 'simple', '', 'relying', 'on', 'the', 'ibm', '', 'data', 'processing', 'system', 'for', 'all', 'its', 'processing', 'and', 'output', '', 'since', 'the', 'system', 'has', 'been', 'operating', 'for', 'over', 'a', 'year', '', 'the', 'conclusions', 'drawn', 'are', 'based', 'on', 'actual', 'experience', '']
